# encoding="utf-8"

import re
import os
import requests
import time
import headers

total_page = 600
save_folder = "texts/"
host = '1024.917rbb.info'
domain = 'http://1024.917rbb.info/pw/'
base = 'http://1024.917rbb.info/pw/thread.php?fid=17&page={}'


def main():
    open(save_folder + 'error.txt', "w", encoding="utf8").write("")
    for i in range(1, total_page):
        page_url = base.format(i)
        try:
            page_content = str(
                requests.get(page_url, headers=headers.forum(host), timeout=10).content,
                encoding="utf8",
                errors="ignore"
            )
            if not page_content:
                continue
            detail_urls = get_detail_urls(page_content)
            for url in detail_urls:
                try:
                    content = str(
                        requests.get(domain + url, headers=headers.thread(host), timeout=10).content,
                        encoding="utf8",
                        errors="ignore"
                    )
                    title = get_title(content)
                    text = get_text(content)

                    dir = save_folder + str(i) + '/'
                    if not os.path.exists(dir):
                        os.mkdir(dir)

                    filename = dir + validate_title(title) + '.txt'
                    open(filename, "w", encoding="utf8").write(title + "\n\n" + text)
                    time.sleep(0.5)
                except Exception as e:
                    open(save_folder + 'error.txt', "a", encoding="utf8").write(url + ": " + str(e.args) + "\n")
                    continue
        except Exception as e:
            open(save_folder + 'error.txt', "a", encoding="utf8").write(page_url + ": " + str(e.args) + "\n")
            continue


def validate_title(title):
    rstr = r"[\/\\\:\*\?\"\<\>\|]"  # '/ \ : * ? " < > |'
    new_title = re.sub(rstr, "_", title)  # 替换为下划线
    return new_title


def get_detail_urls(page_content):
    compiles = re.compile(r'<td\sstyle="text-align:left;padding-left:8px".*?<h3><a\shref="(.*?)"\sid="a_ajax_\d{1,10}">.*?<\/a><\/h3>', re.I)
    matches = compiles.findall(page_content)
    return matches


def get_title(content):
    compiles = re.compile(r'<h1\sid="subject_tpc"\sclass="fl">(.*?)</h1>', re.I)
    matches = compiles.findall(content)
    return matches[0].replace("&nbsp;", " ")


def get_text(content):
    compiles = re.compile(r'<div\sclass="tpc_content"\sid="read_tpc">(.*?)</div>', re.I | re.S)
    matches = compiles.findall(content)
    return filter_tags(matches[0].replace('<br>', "\n").replace('<br />', "\n"))


def filter_tags(htmlstr):
    re_h=re.compile('</?\w+[^>]*>')
    s=re_h.sub('',htmlstr)

    s.replace('&nbsp;', '')
    return s


if __name__ == "__main__":
    main()
